#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011  Benjamin Heil
#
# This file is part of Feedmail.
#
# Feedmail is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import feedparser
import logging
import datetime
import time
import socket
import urllib2
import locale
import MySQLdb

from datetime import timedelta
from time import mktime

warn = sys.stderr
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
os_path = os.path.abspath(__file__)
root_path = os.path.join(os.path.split(os.path.split(os_path)[0])[0], 'feedmail')
sys.path.append(root_path)

socket_errors = []
for e in ['error', 'gaierror']:
    if hasattr(socket, e):
        socket_errors.append(getattr(socket, e))

from feedmanager.models import FeedAbo, FeedItem, FeedData, FeedError
from feedmanager.parser import getEntryFingerprint, getEntryFingerprint_MD5, getEntryName, getEntryContent, getEntryBody, contains, getEntryTags, getEntryTitle
import feedmanager.html2text as h2t
from django.conf import settings
from django.core.mail import EmailMessage, mail_admins, EmailMultiAlternatives
from django.template import Context
from django.template.loader import get_template
from django.utils.encoding import smart_unicode

h2t.UNICODE_SNOB = settings.TEXTMODE_UNICODE_SNOB
h2t.LINKS_EACH_PARAGRAPH = settings.TEXTMODE_LINKS_EACH_PARAGRAPH
h2t.BODY_WIDTH = settings.TEXTMODE_BODY_WIDTH
html2text = h2t.html2text

def _add_error_entry(feed, error_code, error_msg, url):
    error = FeedError(feed=feed, error_no=error_code, msg=error_msg, url=url)
    error.save()
    
def _add_logging_error(feed, error_code, error_msg, url, add_error_entry=True):
    if add_error_entry:
        _add_error_entry(feed, error_code, error_msg, url)
    logging.error(u'FEED ERROR: %s\n%s\n%s\n' % (smart_unicode(feed),
                                                 smart_unicode(error_code),
                                                 u"%s (%s)" % (smart_unicode(error_msg), url)))

def send_new_location_mail(feed, old_url):
    body = get_template('mail/new_location.html')
    body_data = Context({
        'old_url': old_url,
        'new_url': feed.url,
        'site_title': settings.SITE_TITLE,
        'site_url': settings.ROOT_URL
    })
    subject = u'[%s] Geänderte Adresse eines Feeds' % settings.SITE_TITLE
    recipients = FeedAbo.objects.filter(feed=feed)
    for recipient in recipients:
        logging.debug(u">> Sending INFO-Mail to %s with subject '%s'" % (recipient.user.email, subject))
        msg = EmailMessage(subject=subject,
                           body=body.render(body_data),
                           to=[recipient.user.email])
        msg.send(fail_silently=False)

def send_feed_gone_mail(feed):
    body = get_template('mail/feed_gone.html')
    body_data = Context({
        'feed_url': feed.url,
        'site_title': settings.SITE_TITLE,
        'site_url': settings.ROOT_URL
    })
    subject = u'[%s] Feed wurde entfernt' % settings.SITE_TITLE
    recipients = FeedAbo.objects.filter(feed=feed)
    for recipient in recipients:
        logging.debug(u">> Sending FEED_GONE-Mail to %s with subject '%s'" % (recipient.user.email, subject))
        msg = EmailMessage(subject=subject,
                           body=body.render(body_data),
                           to=[recipient.user.email])
        msg.send(fail_silently=False)

def send_mail(entry, feed):
    fingerprint = entry.get('id', getEntryFingerprint(entry))
    fingerprint_md5 = getEntryFingerprint_MD5(entry, fingerprint)
    try:
        f = FeedData.objects.get(fingerprint_md5=fingerprint_md5)
        return
    except FeedData.DoesNotExist:
        pass
    try:
        f = FeedData(fingerprint=fingerprint,
                     fingerprint_md5=fingerprint_md5)
        f.save()
    except MySQLdb.IntegrityError, e:
        # Bsp: Duplicate entry 'http://www.corriere.it/rss/Una%20circolare%20della%20procura%20d' for key 'fingerprint'
        # Wenn in MySQL ein IntegrityError auftritt, wurde der Fingerprint aufgrund der UTF-8-Codierung beim vorherigen
        # Durchlauf abgeschnitten und wird deshalb bei der Suche nicht gefunden. Die Mail für diesen Eintrag ging aber
        # schon raus. Um einen weiteren Mail-Versand zu verhindern, verlassen wir hier die Funktion.
        logging.error('IntegrityError when saving fingerprint "%s" from feed "%s": %s' % (fingerprint, feed, e))
        return
    title = getEntryTitle(entry)
    entry_datetime = time.gmtime()
    for datetype in ('modified', 'issued', 'created'):
        kind = datetype + "_parsed"
        if kind in entry and entry[kind]:
            entry_datetime = entry[kind]

    name = getEntryName(feed, entry)
    from_addr = '"%s" <%s>' % (name, settings.DEFAULT_FROM_EMAIL)
    extra_headers = {
        'Date': time.strftime("%a, %d %b %Y %H:%M:%S -0000", entry_datetime),
        'User-Agent': '%s -- %s' % (settings.SITE_TITLE, settings.ROOT_URL),
        'X-RSS-Feed': feed.url,
        'X-RSS-ID': fingerprint,
        'X-RSS-Tags': getEntryTags(entry)
    }
    try:
        body = getEntryBody(entry, title)
    except UnicodeDecodeError:
        # Dieses Exception ist bisher nur einmal aufgetreten. Im Feed war folgender Titel:
        #   <title>Akismet 2.5 for WordPress Released &#8211; Film at 11</title>
        # Das kracht aber erst beim Zusammensetzen des Bodys - und auch nur in der Produktionsumgebung
        # Daher erstmal diese Exception abfangen und diesen Eintrag ueberspringen
        logging.error(u'UnicodeDecodeError when parsing feed "%s" (entry with fingerprint "%s")' % (feed, fingerprint))
        return

    text_body = html2text(body)
    recipients = FeedAbo.objects.filter(feed=feed)
    for recipient in recipients:
        logging.debug(u">> Sending Mail to %s with subject '%s'" % (recipient.user.email, title))
        msg = EmailMultiAlternatives(subject=title,
                                     body=text_body,
                                     from_email=from_addr,
                                     to=[recipient.user.email],
                                     headers=extra_headers)
        msg.attach_alternative(body, "text/html")
        msg.send(fail_silently=False)

def process_feed(feed):
    try:
        logging.debug(u"Processing '%s'" % feed.url)
        r = {}
        etag = feed.etag or None
        modified = feed.feed_modified or None
        if modified and isinstance(modified, datetime.datetime):
            modified = modified.timetuple()
        try:
            r = feedparser.parse(feed.url, etag=etag, modified=modified, agent=settings.FEEDPARSER_USER_AGENT)
        except UnicodeDecodeError:
            logging.error(u'UnicodeDecodeError when parsing feed "%s"' % (feed))
            return
        http_status = r.get('status', 200)
        http_headers = r.get('headers', {'content-type': 'application/rss+xml', 'content-length':'1'})
        exc_type = r.get("bozo_exception", Exception()).__class__
        if http_status != 304 and http_status != 302 and not r.get('version', ''):
            if http_status not in [200, 302, 404]:
                _add_logging_error(feed, http_status, u'Unbekannter Fehler', feed.url)
            elif http_status == 404:
                _add_logging_error(feed, http_status, u'URL nicht gefunden', feed.url)
            elif http_headers.get('content-type', 'rss').find('html') != -1:
                _add_logging_error(feed, http_status, u'Sieht nach HTML aus!', feed.url)
            elif http_headers.get('content-length', '1') == '0':
                _add_logging_error(feed, http_status, u'Leere Seite', feed.url)
            elif hasattr(socket, 'timeout') and exc_type == socket.timeout:
                _add_logging_error(feed, http_status, u'Feed brauchte zu lange zum Antworten', feed.url)
            elif exc_type == IOError:
                _add_logging_error(feed, http_status, r.bozo_exception, feed.url)
            elif hasattr(feedparser, 'zlib') and exc_type == feedparser.zlib.error:
                _add_logging_error(feed, http_status, u'Fehlerhafte Kompression des Feeds', feed.url)
            elif exc_type in socket_errors:
                _add_logging_error(feed, http_status, r.bozo_exception.args[1], feed.url)
            elif exc_type == urllib2.URLError:
                if r.bozo_exception.reason.__class__ in socket_errors:
                    exc_reason = r.bozo_exception.reason.args[1]
                else:
                    exc_reason = r.bozo_exception.reason
                _add_logging_error(feed, http_status, exc_reason, feed.url)
            elif exc_type == AttributeError:
                _add_logging_error(feed, http_status, r.bozo_exception, feed.url)
            elif exc_type == KeyboardInterrupt:
                raise r.bozo_exception
            elif r.bozo:
                _add_logging_error(feed, http_status, r.get("bozo_exception", u"kann nicht verarbeiten"), feed.url)
            return
        if http_status == 301: # Moved permanently
            _old_url = feed.url
            feed.url = r.get('href', feed.url)
            feed.save()
            send_new_location_mail(feed, _old_url)
            warning_msg = "FEED MOVED PERMANENTLY, new location at '%s'" % feed.url
            _add_error_entry(feed, http_status, warning_msg, feed.url)
            logging.warning(warning_msg)
            return
        if http_status == 410: # Resource gone
            feed.is_gone = True
            feed.save()
            send_feed_gone_mail(feed)
            warning_msg = "FEED HAS GONE, DISABLED: '%s'" % feed.url
            _add_error_entry(feed, http_status, warning_msg, feed.url)
            logging.warning(warning_msg)
            return
        if http_status == 304: # Not modified
            logging.debug(u"[304] Completed, feed not changed '%s'" % feed.url)
            return
        if http_status == 302: # Temporarily moved
            logging.debug(u"[302] Feed temporarily moved to '%s', will continue to fetch old url" % r.href) 
        r.entries.reverse()
        for entry in r.entries:
            send_mail(entry, feed)
        feed.etag = r.get('etag', None)    
        feed.feed_modified = None
        last_modified = r.get('modified', None)
        if last_modified and isinstance(last_modified, time.struct_time):
            feed.feed_modified = datetime.datetime.fromtimestamp(mktime(last_modified))
        feed.save()
        logging.debug(u"[%d] Completed '%s'" % (http_status, feed.url))
    except Exception, e:
        logging.error('Exception in process_feed for feed "%s"' % feed.url)
        logging.exception(e)
        if settings.CRON_SEND_EXCEPTION_MAILS:
            mail_admins('Exception in process_feed for feed "%s"' % feed.url, e)

def clean_feederrors_up():
    """Delete all FeedError-entries with a timestamp older than in settings
    defined number of weeks."""
    logging.debug('Will delete FeedError-entries older than %s weeks ...' % settings.FEEDERROR_WEEKS)
    d = datetime.datetime.now() - datetime.timedelta(weeks=settings.FEEDERROR_WEEKS)
    f = FeedError.objects.filter(timestamp__lte=d)
    logging.debug('%s entries to delete ...' % len(f))
    f.delete()
    logging.debug('FeedError-entries deleted')
    
def set_US_time_locale():
    """Set time locale to en_US.utf-8. That's important for time.strftime,
    because Date should be in english format for Thunderbird and other
    mail programs.
    IMPORTANT: This locale must be available. Check with 'locale -a'!
    """
    locale.setlocale(locale.LC_TIME, ('en_US', 'utf-8'))
        
def run_cron():
    logging.basicConfig(filename=settings.CRON_LOG_FILE,
                level=settings.CRON_LOG_LEVEL,
                format='%(asctime)s %(levelname)-8s %(message)s')
    try:
        starttime = datetime.datetime.now()
        logging.warn('feedmail cron begins')
        set_US_time_locale()
        feeds = FeedItem.objects.all()
        for feed in feeds:
            if not feed.is_gone:
                process_feed(feed)
        clean_feederrors_up()
        run_time = datetime.datetime.now() - starttime
        logging.warn('feedmail cron ends (elapsed %s)' % run_time)
        if run_time > timedelta(minutes=settings.CRON_SEND_WARNING_AFTER_MIN):
            msg = 'Warning: feedmail cron job takes %s to run (> %s min)' % (run_time, settings.CRON_SEND_WARNING_AFTER_MIN)
            mail_admins(msg, msg)
    except Exception, e:
        logging.error('General exception in feedmail_cron:')
        logging.exception(e)
        if settings.CRON_SEND_EXCEPTION_MAILS:
            mail_admins('General exception in feedmail_cron, check log file', e)

if __name__ == '__main__':
    run_cron()
